/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package treccontextsug;

import de.l3s.boilerpipe.BoilerpipeProcessingException;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.xml.sax.SAXException;

/**
 *
 * @author marcelo
 */
public class ReadSubcollectionJSON {

    private String subcollectionPath;

    public ReadSubcollectionJSON(String subcollectionPath) {
        this.subcollectionPath = subcollectionPath;
    }
    private String[] blackList = {"Last-Modified:", "ETag:", "Accept-Ranges:", "Content-Length:",
        "Content-Type:", "User-Agent", "Accept-Encoding", "X-Powered-By:", "HTTP", "Set-Cookie:", "Connection:", "OFFICE PHONE:", "P.O.Box",
        "p.m.", "a.m."};

    public void fillLuceneSearcher(Context context, LuceneSearch search) throws IOException, SAXException, BoilerpipeProcessingException, ParseException {
        System.out.println("Inserting atttractions for " + context.getCity() + "-" + context.getState());
        String inputJsonFile = subcollectionPath + context.getCity().toLowerCase().replaceAll("[\\., ]", "") + "-" + context.getState().toLowerCase() + ".json";

        // open our gzip input stream
        //GZIPInputStream gzInputStream = new GZIPInputStream(new FileInputStream(inputJsonFile));
        InputStream gzInputStream = new FileInputStream(inputJsonFile);

        JSONParser parser = new JSONParser();
        JSONArray jsonArray = (JSONArray) parser.parse(new InputStreamReader(gzInputStream));

        Iterator i = jsonArray.iterator();
        while (i.hasNext()) {
            JSONObject j = (JSONObject) i.next();
            JSONObject metadata = (JSONObject) j.get("metadata");
            if (metadata == null) {
                continue;
            }

            String name = (String) metadata.get("name");
            name = name.replaceAll("[^\\p{ASCII}]", "");
            if (name.length() > 64) {
                name = name.substring(0, 64);
            }
            name = name.replace('\"', '\'');
            if (name == null) {
                name = "";
            }

            JSONArray types = (JSONArray) metadata.get("types");
            String baseNames = "";
            if (types != null) {
                Iterator t = types.iterator();
                while (t.hasNext()) {
                    String type = (String) t.next();
                    baseNames += type + ",";
                }
                baseNames = baseNames.substring(0, baseNames.length() - 1);
            }

            String docId = (String) j.get("docid");
            String url = (String) j.get("url");

            String documentContent = "";
            String description = "";

            JSONArray descr = (JSONArray) j.get("description");
            Iterator k = ((JSONArray) descr.get(0)).iterator();
            while (k.hasNext()) {
                String doc = (String) k.next();
                doc = doc.replaceAll("\\A\\s+(.*?)\\s+\\z", "$1");
                doc = doc.replaceAll("\\s+", " ");
                doc = doc.replaceAll("[^\\p{ASCII}]", "");

                if (description.equals("")) {
                    if (doc.toCharArray().length < 512) {
                        boolean notGood = false;
                        for (String black : blackList) {
                            if (doc.contains(black)) {
                                notGood = true;
                                break;
                            }
                        }
                        if (!notGood) {
                            description = doc.replace('\"', '\'');
                        }
                    }
                }

                documentContent += doc + " ";
            }

            if (description.equals("")) {
                if (!name.equals("")) {
                    description = name;
                } else {
                    description = "Description";
                    name = "Title";
                }
            } else {
                if (name.equals("")) {
                    if (description.length() > 64) {
                        name = description.substring(0, 64);
                    } else {
                        name = description;
                    }
                }
            }
            search.addDocument(docId, url, name, description, baseNames, documentContent);
        }
        gzInputStream.close();
    }

    public void fillLuceneSearcherOld(Context context, LuceneSearch search) throws IOException, SAXException, BoilerpipeProcessingException, ParseException {
        System.out.println("Inserting atttractions for " + context.getCity() + "-" + context.getState());
        String inputJsonFile = subcollectionPath + context.getCity().toLowerCase().replaceAll("[\\., ]", "") + "-" + context.getState().toLowerCase() + "2.json.gz";

        // open our gzip input stream
        GZIPInputStream gzInputStream = new GZIPInputStream(new FileInputStream(inputJsonFile));
        //InputStream gzInputStream = new FileInputStream(subcollectionPath + "albany-ny.json");

        JSONParser parser = new JSONParser();
        JSONObject jsonObject = (JSONObject) parser.parse(new InputStreamReader(gzInputStream));

        JSONArray s = (JSONArray) jsonObject.get("x");
        Iterator i = s.iterator();
        while (i.hasNext()) {
            JSONObject j = (JSONObject) i.next();
            String docId = (String) j.get("docid");
            String url = (String) j.get("url");
            String documentContent = "";

            String title = (String) j.get("title");
            String description = (String) j.get("abstract");

            //System.out.println(docId);
            JSONArray descr = (JSONArray) j.get("description");
            Iterator k = descr.iterator();
            while (k.hasNext()) {
                String doc = (String) k.next();
                documentContent += doc + " ";
            }

            search.addDocument(docId, url, title, description, "", documentContent);
        }
        gzInputStream.close();
    }

    public static void main(String[] args) throws IOException, ParseException, SAXException, BoilerpipeProcessingException {
        ReadSubcollectionJSON reader = new ReadSubcollectionJSON("/Users/manzato/Databases/TREC/clueweb12-descriptions/");
        ContextsSet contextReader = new ContextsSet("/Users/manzato/Databases/TREC/contexts2013.json");
        contextReader.readJSON();

        reader.addTitle(contextReader);
    }

    public void addTitle(ContextsSet contexts) throws IOException, SAXException, BoilerpipeProcessingException, ParseException {
        for (Context context : contexts.getAllContexts()) {
            System.out.println("Inserting atttractions for " + context.getCity() + "-" + context.getState());
            String inputJsonFile = subcollectionPath + context.getCity().toLowerCase().replaceAll("[\\., ]", "") + "-" + context.getState().toLowerCase() + ".json.gz";
            String outputJsonFile = subcollectionPath + context.getCity().toLowerCase().replaceAll("[\\., ]", "") + "-" + context.getState().toLowerCase() + "2.json.gz";

            // open our gzip input stream
            GZIPInputStream gzInputStream = new GZIPInputStream(new FileInputStream(inputJsonFile));
            GZIPOutputStream gzOutputStream = new GZIPOutputStream(new FileOutputStream(outputJsonFile));

            JSONParser parser = new JSONParser();
            JSONObject jsonObject = (JSONObject) parser.parse(new InputStreamReader(gzInputStream));

            JSONObject rootOut = new JSONObject();

            JSONArray s = (JSONArray) jsonObject.get("x");
            JSONArray sOut = new JSONArray();

            Iterator i = s.iterator();
            while (i.hasNext()) {
                JSONObject j = (JSONObject) i.next();
                JSONObject jOut = new JSONObject();

                String docId = (String) j.get("docid");
                jOut.put("docid", docId);

                String url = (String) j.get("url");
                jOut.put("url", url);

                String title = getPageTitle(new URL(url));
                jOut.put("title", title);

                String description = "";
                JSONArray descr = (JSONArray) j.get("description");
                JSONArray descrOut = new JSONArray();

                Iterator k = descr.iterator();
                while (k.hasNext()) {
                    String doc = (String) k.next();
                    descrOut.add(doc);

                    if (description.equals("")) {
                        if (doc.length() <= 512) {
                            description = doc;
                        }
                    }
                }
                jOut.put("abstract", description);
                jOut.put("description", descrOut);

                sOut.add(jOut);
                //break;
            }
            rootOut.put("x", sOut);

            String string = rootOut.toJSONString();
            gzOutputStream.write(string.getBytes());

            gzInputStream.close();
            gzOutputStream.close();
        }
    }

    String getPageTitle(URL url) {
        String title = "";

        try {
            HttpURLConnection httpcon = (HttpURLConnection) url.openConnection();
            httpcon.addRequestProperty("User-Agent", "Mozilla/4.76");
            BufferedReader in = new BufferedReader(new InputStreamReader(httpcon.getInputStream()));

            Pattern pHead = Pattern.compile("(?i)</HEAD>");
            Matcher mHead;
            Pattern pTitle = Pattern.compile("(?i)</TITLE>");
            Matcher mTitle;

            String inputLine;
            boolean found = false;
            boolean notFound = false;
            String html = "";
            while (!(((inputLine = in.readLine()) == null) || found || notFound)) {
                html = html + inputLine;
                mHead = pHead.matcher(inputLine);
                if (mHead.find()) {
                    notFound = true;
                } else {
                    mTitle = pTitle.matcher(inputLine);
                    if (mTitle.find()) {
                        found = true;
                        //System.out.println(inputLine);
                    }
                }
            }
            in.close();

            html = html.replaceAll("\\s+", " ");
            if (found) {
                Pattern p = Pattern.compile("(?i)<TITLE.*?>(.*?)</TITLE>");
                Matcher m = p.matcher(html);
                while (m.find() == true) {
                    title = m.group(1);
                    //System.out.println("Title "+title); 
                }
            }
        } catch (IOException e) {
            System.out.println("Exception");
            title = "";
        }
        if (title.length() > 64) {
            return title.substring(0, 64);
        }
        return title;
    }
}
